{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.layers import Embedding\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Conv1D, MaxPooling1D, AveragePooling1D, GlobalAveragePooling1D\n",
    "from keras.datasets import imdb\n",
    "from keras.models import load_model\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import minmax_scale\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters\n",
    "max_features = 1024\n",
    "batch_size = 64\n",
    "epochs = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load training data\n",
    "train = pd.read_csv('data/train.csv', header=None, sep=',', usecols = list(range(0,max_features)))\n",
    "label = pd.read_csv(\"data/train_label.csv\", sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train.fillna(0)\n",
    "\n",
    "X = X.iloc[:, :]\n",
    "y = label.iloc[:, -1]\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.05, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (107954, 1024)\n",
      "x_test shape: (5682, 1024)\n",
      "y_train shape: (107954,)\n",
      "y_test shape: (5682,)\n"
     ]
    }
   ],
   "source": [
    "print('x_train shape:', x_train.shape)\n",
    "print('x_test shape:', x_test.shape)\n",
    "print('y_train shape:', y_train.shape)\n",
    "print('y_test shape:', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, None, 64)          65536     \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, None, 64)          12352     \n",
      "_________________________________________________________________\n",
      "conv1d_2 (Conv1D)            (None, None, 64)          12352     \n",
      "_________________________________________________________________\n",
      "average_pooling1d_1 (Average (None, None, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv1d_3 (Conv1D)            (None, None, 128)         24704     \n",
      "_________________________________________________________________\n",
      "conv1d_4 (Conv1D)            (None, None, 128)         49280     \n",
      "_________________________________________________________________\n",
      "average_pooling1d_2 (Average (None, None, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv1d_5 (Conv1D)            (None, None, 256)         98560     \n",
      "_________________________________________________________________\n",
      "conv1d_6 (Conv1D)            (None, None, 256)         196864    \n",
      "_________________________________________________________________\n",
      "average_pooling1d_3 (Average (None, None, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv1d_7 (Conv1D)            (None, None, 512)         393728    \n",
      "_________________________________________________________________\n",
      "conv1d_8 (Conv1D)            (None, None, 512)         786944    \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d_1 ( (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 513       \n",
      "=================================================================\n",
      "Total params: 1,640,833\n",
      "Trainable params: 1,640,833\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 107954 samples, validate on 5682 samples\n",
      "Epoch 1/40\n",
      "107954/107954 [==============================] - 112s 1ms/step - loss: 0.2387 - acc: 0.8947 - val_loss: 0.1407 - val_acc: 0.9483\n",
      "Epoch 2/40\n",
      "107954/107954 [==============================] - 111s 1ms/step - loss: 0.1324 - acc: 0.9499 - val_loss: 0.1272 - val_acc: 0.9535\n",
      "Epoch 3/40\n",
      "107954/107954 [==============================] - 113s 1ms/step - loss: 0.1106 - acc: 0.9599 - val_loss: 0.0979 - val_acc: 0.9652\n",
      "Epoch 4/40\n",
      "107954/107954 [==============================] - 114s 1ms/step - loss: 0.0991 - acc: 0.9642 - val_loss: 0.0949 - val_acc: 0.9660\n",
      "Epoch 5/40\n",
      "107954/107954 [==============================] - 115s 1ms/step - loss: 0.0901 - acc: 0.9679 - val_loss: 0.0875 - val_acc: 0.9669\n",
      "Epoch 6/40\n",
      "107954/107954 [==============================] - 115s 1ms/step - loss: 0.0814 - acc: 0.9712 - val_loss: 0.0900 - val_acc: 0.9722\n",
      "Epoch 7/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0751 - acc: 0.9737 - val_loss: 0.0813 - val_acc: 0.9725\n",
      "Epoch 8/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0684 - acc: 0.9759 - val_loss: 0.0843 - val_acc: 0.9732\n",
      "Epoch 9/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0626 - acc: 0.9784 - val_loss: 0.0789 - val_acc: 0.9715\n",
      "Epoch 10/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0558 - acc: 0.9809 - val_loss: 0.0778 - val_acc: 0.9748\n",
      "Epoch 11/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0527 - acc: 0.9819 - val_loss: 0.0795 - val_acc: 0.9747\n",
      "Epoch 12/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0490 - acc: 0.9833 - val_loss: 0.0727 - val_acc: 0.9752\n",
      "Epoch 13/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0429 - acc: 0.9857 - val_loss: 0.0832 - val_acc: 0.9741\n",
      "Epoch 14/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0398 - acc: 0.9865 - val_loss: 0.0871 - val_acc: 0.9743\n",
      "Epoch 15/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0364 - acc: 0.9878 - val_loss: 0.0914 - val_acc: 0.9743\n",
      "Epoch 16/40\n",
      "107954/107954 [==============================] - 115s 1ms/step - loss: 0.0352 - acc: 0.9880 - val_loss: 0.0958 - val_acc: 0.9720\n",
      "Epoch 17/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0314 - acc: 0.9896 - val_loss: 0.0871 - val_acc: 0.9738\n",
      "Epoch 18/40\n",
      "107954/107954 [==============================] - 115s 1ms/step - loss: 0.0291 - acc: 0.9907 - val_loss: 0.0932 - val_acc: 0.9748\n",
      "Epoch 19/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0280 - acc: 0.9907 - val_loss: 0.0887 - val_acc: 0.9787\n",
      "Epoch 20/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0251 - acc: 0.9915 - val_loss: 0.1049 - val_acc: 0.9732\n",
      "Epoch 21/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0237 - acc: 0.9923 - val_loss: 0.1048 - val_acc: 0.9757\n",
      "Epoch 22/40\n",
      "107954/107954 [==============================] - 117s 1ms/step - loss: 0.0232 - acc: 0.9921 - val_loss: 0.0936 - val_acc: 0.9724\n",
      "Epoch 23/40\n",
      "107954/107954 [==============================] - 117s 1ms/step - loss: 0.0216 - acc: 0.9930 - val_loss: 0.1081 - val_acc: 0.9752\n",
      "Epoch 24/40\n",
      "107954/107954 [==============================] - 117s 1ms/step - loss: 0.0210 - acc: 0.9931 - val_loss: 0.0965 - val_acc: 0.9764\n",
      "Epoch 25/40\n",
      "107954/107954 [==============================] - 117s 1ms/step - loss: 0.0203 - acc: 0.9933 - val_loss: 0.1139 - val_acc: 0.9755\n",
      "Epoch 26/40\n",
      "107954/107954 [==============================] - 117s 1ms/step - loss: 0.0196 - acc: 0.9938 - val_loss: 0.1076 - val_acc: 0.9743\n",
      "Epoch 27/40\n",
      "107954/107954 [==============================] - 117s 1ms/step - loss: 0.0189 - acc: 0.9936 - val_loss: 0.1128 - val_acc: 0.9761\n",
      "Epoch 28/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0189 - acc: 0.9937 - val_loss: 0.1015 - val_acc: 0.9759\n",
      "Epoch 29/40\n",
      "107954/107954 [==============================] - 117s 1ms/step - loss: 0.0171 - acc: 0.9945 - val_loss: 0.1111 - val_acc: 0.9734\n",
      "Epoch 30/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0181 - acc: 0.9941 - val_loss: 0.1020 - val_acc: 0.9766\n",
      "Epoch 31/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0149 - acc: 0.9951 - val_loss: 0.1151 - val_acc: 0.9773\n",
      "Epoch 32/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0169 - acc: 0.9946 - val_loss: 0.1029 - val_acc: 0.9775\n",
      "Epoch 33/40\n",
      "107954/107954 [==============================] - 116s 1ms/step - loss: 0.0150 - acc: 0.9952 - val_loss: 0.1188 - val_acc: 0.9738\n",
      "Epoch 34/40\n",
      "107954/107954 [==============================] - 117s 1ms/step - loss: 0.0159 - acc: 0.9946 - val_loss: 0.1243 - val_acc: 0.9724\n",
      "Epoch 35/40\n",
      "107954/107954 [==============================] - 117s 1ms/step - loss: 0.0137 - acc: 0.9956 - val_loss: 0.1080 - val_acc: 0.9769\n",
      "Epoch 36/40\n",
      "107954/107954 [==============================] - 117s 1ms/step - loss: 0.0149 - acc: 0.9953 - val_loss: 0.1153 - val_acc: 0.9766\n",
      "Epoch 37/40\n",
      "107954/107954 [==============================] - 118s 1ms/step - loss: 0.0142 - acc: 0.9956 - val_loss: 0.1215 - val_acc: 0.9748\n",
      "Epoch 38/40\n",
      "107954/107954 [==============================] - 118s 1ms/step - loss: 0.0141 - acc: 0.9957 - val_loss: 0.1204 - val_acc: 0.9731\n",
      "Epoch 39/40\n",
      "107954/107954 [==============================] - 118s 1ms/step - loss: 0.0132 - acc: 0.9960 - val_loss: 0.1337 - val_acc: 0.9741\n",
      "Epoch 40/40\n",
      "107954/107954 [==============================] - 118s 1ms/step - loss: 0.0131 - acc: 0.9958 - val_loss: 0.1344 - val_acc: 0.9754\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1f004f49da0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# VGG current best\n",
    "#del model\n",
    "model = Sequential()\n",
    "model.add(Embedding(max_features, 64)) #128:no\n",
    "model.add(Conv1D(64, 3, activation='relu', strides=1, padding='valid'))\n",
    "model.add(Conv1D(64, 3, activation='relu'))\n",
    "model.add(AveragePooling1D(3))\n",
    "model.add(Conv1D(128, 3, activation='relu')) \n",
    "model.add(Conv1D(128, 3, activation='relu'))\n",
    "model.add(AveragePooling1D(3))\n",
    "model.add(Conv1D(256, 3, activation='relu'))\n",
    "model.add(Conv1D(256, 3, activation='relu'))\n",
    "model.add(AveragePooling1D(3))\n",
    "model.add(Conv1D(512, 3, activation='relu'))\n",
    "model.add(Conv1D(512, 3, activation='relu'))\n",
    "model.add(GlobalAveragePooling1D())\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(1, activation='sigmoid'))  # tanh:no \n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "model.summary()\n",
    "\n",
    "model.fit(x_train, y_train, batch_size=batch_size, epochs=40, verbose=1, validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, None, 64)          65536     \n",
      "_________________________________________________________________\n",
      "conv1d_9 (Conv1D)            (None, None, 64)          12352     \n",
      "_________________________________________________________________\n",
      "conv1d_10 (Conv1D)           (None, None, 64)          12352     \n",
      "_________________________________________________________________\n",
      "average_pooling1d_1 (Average (None, None, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv1d_11 (Conv1D)           (None, None, 128)         24704     \n",
      "_________________________________________________________________\n",
      "conv1d_12 (Conv1D)           (None, None, 128)         49280     \n",
      "_________________________________________________________________\n",
      "average_pooling1d_2 (Average (None, None, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv1d_13 (Conv1D)           (None, None, 256)         98560     \n",
      "_________________________________________________________________\n",
      "conv1d_14 (Conv1D)           (None, None, 256)         196864    \n",
      "_________________________________________________________________\n",
      "average_pooling1d_3 (Average (None, None, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv1d_15 (Conv1D)           (None, None, 512)         393728    \n",
      "_________________________________________________________________\n",
      "conv1d_16 (Conv1D)           (None, None, 512)         786944    \n",
      "_________________________________________________________________\n",
      "conv1d_17 (Conv1D)           (None, None, 512)         786944    \n",
      "_________________________________________________________________\n",
      "global_average_pooling1d_2 ( (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 513       \n",
      "=================================================================\n",
      "Total params: 2,427,777\n",
      "Trainable params: 2,427,777\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 107954 samples, validate on 5682 samples\n",
      "Epoch 1/40\n",
      "107954/107954 [==============================] - 123s 1ms/step - loss: 0.2772 - acc: 0.8656 - val_loss: 0.1340 - val_acc: 0.9476\n",
      "Epoch 2/40\n",
      "107954/107954 [==============================] - 125s 1ms/step - loss: 0.1394 - acc: 0.9473 - val_loss: 0.1417 - val_acc: 0.9493\n",
      "Epoch 3/40\n",
      "107954/107954 [==============================] - 127s 1ms/step - loss: 0.1174 - acc: 0.9570 - val_loss: 0.1152 - val_acc: 0.9585\n",
      "Epoch 4/40\n",
      "107954/107954 [==============================] - 127s 1ms/step - loss: 0.1041 - acc: 0.9624 - val_loss: 0.1171 - val_acc: 0.9599\n",
      "Epoch 5/40\n",
      "107954/107954 [==============================] - 127s 1ms/step - loss: 0.0947 - acc: 0.9666 - val_loss: 0.1176 - val_acc: 0.9602\n",
      "Epoch 6/40\n",
      "107954/107954 [==============================] - 127s 1ms/step - loss: 0.0857 - acc: 0.9699 - val_loss: 0.0898 - val_acc: 0.9694\n",
      "Epoch 7/40\n",
      "107954/107954 [==============================] - 127s 1ms/step - loss: 0.0787 - acc: 0.9728 - val_loss: 0.0857 - val_acc: 0.9706\n",
      "Epoch 8/40\n",
      "107954/107954 [==============================] - 128s 1ms/step - loss: 0.0717 - acc: 0.9752 - val_loss: 0.0891 - val_acc: 0.9671\n",
      "Epoch 9/40\n",
      "107954/107954 [==============================] - 129s 1ms/step - loss: 0.0656 - acc: 0.9772 - val_loss: 0.0808 - val_acc: 0.9715\n",
      "Epoch 10/40\n",
      "107954/107954 [==============================] - 128s 1ms/step - loss: 0.0609 - acc: 0.9791 - val_loss: 0.0833 - val_acc: 0.9699\n",
      "Epoch 11/40\n",
      "107954/107954 [==============================] - 128s 1ms/step - loss: 0.0563 - acc: 0.9807 - val_loss: 0.0927 - val_acc: 0.9699\n",
      "Epoch 12/40\n",
      "107954/107954 [==============================] - 128s 1ms/step - loss: 0.0504 - acc: 0.9829 - val_loss: 0.0856 - val_acc: 0.9715\n",
      "Epoch 13/40\n",
      "107954/107954 [==============================] - 128s 1ms/step - loss: 0.0474 - acc: 0.9839 - val_loss: 0.0834 - val_acc: 0.9720\n",
      "Epoch 14/40\n",
      "107954/107954 [==============================] - 129s 1ms/step - loss: 0.0426 - acc: 0.9855 - val_loss: 0.0799 - val_acc: 0.9725\n",
      "Epoch 15/40\n",
      "107954/107954 [==============================] - 129s 1ms/step - loss: 0.0407 - acc: 0.9861 - val_loss: 0.0867 - val_acc: 0.9732\n",
      "Epoch 16/40\n",
      "107954/107954 [==============================] - 130s 1ms/step - loss: 0.0371 - acc: 0.9876 - val_loss: 0.0823 - val_acc: 0.9734\n",
      "Epoch 17/40\n",
      "107954/107954 [==============================] - 131s 1ms/step - loss: 0.0351 - acc: 0.9885 - val_loss: 0.1096 - val_acc: 0.9694\n",
      "Epoch 18/40\n",
      "107954/107954 [==============================] - 131s 1ms/step - loss: 0.0331 - acc: 0.9889 - val_loss: 0.0980 - val_acc: 0.9729\n",
      "Epoch 19/40\n",
      "107954/107954 [==============================] - 131s 1ms/step - loss: 0.0299 - acc: 0.9902 - val_loss: 0.0858 - val_acc: 0.9752\n",
      "Epoch 20/40\n",
      "107954/107954 [==============================] - 130s 1ms/step - loss: 0.0283 - acc: 0.9903 - val_loss: 0.0896 - val_acc: 0.9754\n",
      "Epoch 21/40\n",
      "107954/107954 [==============================] - 131s 1ms/step - loss: 0.0270 - acc: 0.9912 - val_loss: 0.1069 - val_acc: 0.9715\n",
      "Epoch 22/40\n",
      "107954/107954 [==============================] - 131s 1ms/step - loss: 0.0252 - acc: 0.9915 - val_loss: 0.1134 - val_acc: 0.9727\n",
      "Epoch 23/40\n",
      "107954/107954 [==============================] - 131s 1ms/step - loss: 0.0238 - acc: 0.9923 - val_loss: 0.0982 - val_acc: 0.9766\n",
      "Epoch 24/40\n",
      "107954/107954 [==============================] - 130s 1ms/step - loss: 0.0225 - acc: 0.9928 - val_loss: 0.0994 - val_acc: 0.9732\n",
      "Epoch 25/40\n",
      "107954/107954 [==============================] - 130s 1ms/step - loss: 0.0207 - acc: 0.9933 - val_loss: 0.1037 - val_acc: 0.9740\n",
      "Epoch 26/40\n",
      "107954/107954 [==============================] - 130s 1ms/step - loss: 0.0212 - acc: 0.9930 - val_loss: 0.1026 - val_acc: 0.9729\n",
      "Epoch 27/40\n",
      "107954/107954 [==============================] - 130s 1ms/step - loss: 0.0197 - acc: 0.9937 - val_loss: 0.0839 - val_acc: 0.9759\n",
      "Epoch 28/40\n",
      "107954/107954 [==============================] - 129s 1ms/step - loss: 0.0188 - acc: 0.9936 - val_loss: 0.1113 - val_acc: 0.9731\n",
      "Epoch 29/40\n",
      "107954/107954 [==============================] - 129s 1ms/step - loss: 0.0187 - acc: 0.9941 - val_loss: 0.0949 - val_acc: 0.9747\n",
      "Epoch 30/40\n",
      "107954/107954 [==============================] - 128s 1ms/step - loss: 0.0170 - acc: 0.9948 - val_loss: 0.0790 - val_acc: 0.9768\n",
      "Epoch 31/40\n",
      "107954/107954 [==============================] - 129s 1ms/step - loss: 0.0168 - acc: 0.9948 - val_loss: 0.0909 - val_acc: 0.9752\n",
      "Epoch 32/40\n",
      "107954/107954 [==============================] - 128s 1ms/step - loss: 0.0167 - acc: 0.9946 - val_loss: 0.1032 - val_acc: 0.9755\n",
      "Epoch 33/40\n",
      "107954/107954 [==============================] - 128s 1ms/step - loss: 0.0166 - acc: 0.9949 - val_loss: 0.1187 - val_acc: 0.9748\n",
      "Epoch 34/40\n",
      "107954/107954 [==============================] - 128s 1ms/step - loss: 0.0156 - acc: 0.9950 - val_loss: 0.1219 - val_acc: 0.9743\n",
      "Epoch 35/40\n",
      "107954/107954 [==============================] - 128s 1ms/step - loss: 0.0151 - acc: 0.9951 - val_loss: 0.1080 - val_acc: 0.9738\n",
      "Epoch 36/40\n",
      "107954/107954 [==============================] - 129s 1ms/step - loss: 0.0149 - acc: 0.9955 - val_loss: 0.0951 - val_acc: 0.9778\n",
      "Epoch 37/40\n",
      "107954/107954 [==============================] - 129s 1ms/step - loss: 0.0137 - acc: 0.9957 - val_loss: 0.1257 - val_acc: 0.9717\n",
      "Epoch 38/40\n",
      "107954/107954 [==============================] - 129s 1ms/step - loss: 0.0146 - acc: 0.9953 - val_loss: 0.1072 - val_acc: 0.9736\n",
      "Epoch 39/40\n",
      "107954/107954 [==============================] - 129s 1ms/step - loss: 0.0140 - acc: 0.9955 - val_loss: 0.1181 - val_acc: 0.9748\n",
      "Epoch 40/40\n",
      "107954/107954 [==============================] - 129s 1ms/step - loss: 0.0143 - acc: 0.9954 - val_loss: 0.1055 - val_acc: 0.9757\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2310c301278>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# VGG new trial\n",
    "del model\n",
    "model = Sequential()\n",
    "model.add(Embedding(max_features, 64)) #128:no\n",
    "model.add(Conv1D(64, 3, activation='relu', strides=1, padding='valid'))\n",
    "model.add(Conv1D(64, 3, activation='relu'))\n",
    "model.add(AveragePooling1D(3))\n",
    "model.add(Conv1D(128, 3, activation='relu')) \n",
    "model.add(Conv1D(128, 3, activation='relu'))\n",
    "model.add(AveragePooling1D(3))\n",
    "model.add(Conv1D(256, 3, activation='relu'))\n",
    "model.add(Conv1D(256, 3, activation='relu'))\n",
    "model.add(AveragePooling1D(3))\n",
    "model.add(Conv1D(512, 3, activation='relu'))\n",
    "model.add(Conv1D(512, 3, activation='relu'))\n",
    "model.add(Conv1D(512, 3, activation='relu'))\n",
    "model.add(GlobalAveragePooling1D())\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(1, activation='sigmoid'))  # tanh:no \n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "model.summary()\n",
    "\n",
    "model.fit(x_train, y_train, batch_size=batch_size, epochs=40, verbose=1, validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 107954 samples, validate on 5682 samples\n",
      "Epoch 1/1\n",
      "107954/107954 [==============================] - 120s 1ms/step - loss: 0.0128 - acc: 0.9962 - val_loss: 0.1066 - val_acc: 0.9776\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2315c419f98>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# model.save('model/CNN_LSTM_model.h5')\n",
    "# del model\n",
    "# model = load_model('model/CNN_LSTM_model.h5')\n",
    "model.fit(x_train, y_train, batch_size=batch_size, epochs=1, verbose=1, validation_data=(x_test, y_test))\n",
    "#print(model.metrics_names)\n",
    "#score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)\n",
    "#print('Test score:', score)\n",
    "#print('Test accuracy:', acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv('data/test.csv', header=None, names = list(range(0,max_features)))\n",
    "test = test.fillna(0)\n",
    "\n",
    "test = test.iloc[:, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit model on test\n",
    "y_pred = model.predict(test, batch_size=batch_size)\n",
    "df = pd.DataFrame(y_pred)\n",
    "df.to_csv(\"data/CNN_prob.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embedding 64 : 0.953537486884343\n",
    "# embedding 128 : 0.9592\n",
    "# VGG V1 - 512 : 0.9740\n",
    "# VGG V1 - 1024: 0.9771\n",
    "# VGG V1 - 1024 - maxpooling: 0.9776"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
